# -*- coding: utf-8 -*-
from scrapy import Spider, Request
from scrapy.selector import Selector
from datetime import datetime, timedelta
import json
import sys
import traceback
import urllib3
from datetime import datetime
from pymongo import MongoClient, UpdateOne,ASCENDING
import pandas as pd
import time

class TaobaoSpider(Spider):
    name = 'stock_daily'
    db = MongoClient('mongodb://39.107.106.234:27017')['finaa_quant']
    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'
    op=0
    def start_requests(self,date=None):

        hfq_daily_cursor = self.db['daily_index_250'].find(
            {'code': '000001', },
            sort=[('date', ASCENDING)],
            projection={'date': True, 'close': True})

        self.db['daily_index_250'].remove({'date': hfq_daily_cursor[0]['date']})
        self.db['daily_k_250'].remove({'date': hfq_daily_cursor[0]['date']})
        self.db['daily_kba_250'].remove({'date': hfq_daily_cursor[0]['date']})

        DB_CONN = MongoClient('mongodb://39.107.106.234:27017')['final_quant']
        datetime_obj = datetime.now()
        if date is None:
            date = datetime_obj.strftime('%Y-%m-%d')
        codes = []

        # while len(codes) == 0:
        #     code_cursor = DB_CONN.basic.find(
        #         {'date': date},
        #         projection={'code': True, '_id': False})
        #
        #     codes = [x['code'] for x in code_cursor]
        #
        #     datetime_obj = datetime_obj - timedelta(days=1)
        #     date = datetime_obj.strftime('%Y-%m-%d')
        # url = 'http://pdfm2.eastmoney.com/EM_UBG_PDTI_Fast/api/js?id={}&TYPE=r&rtntype=5&isCR=false&token=4d2cd9e55b669a2b3dec49542914281d&js=cb_{}&authorityType={}'
        # for asa in codes[1:]:
        #     if asa[0:2] == '60':
        #         asa=asa+'1'
        #     elif asa[0:2] in ['30', '00']:
        #         asa=asa+'2'
        #     time_stamp = time.time()
        #     digits = 10 ** (13 - 10)
        #     time_stamp = int(round(time_stamp * digits))
        #
        #
        #     yield Request(url.format(asa,time_stamp,type),self.dual,meta={'code':asa,'type':'k'})
        #     break

        index_codes = ['000001', '000300']
        indexurl = 'http://pdfm.eastmoney.com/EM_UBG_PDTI_Fast/api/js?rtntype=1&token=4f1862fc3b5e77c150a2b985b12db0fd&cb=&id={0}1&type=k&iscr=false&_=1528087227767'
        for index in index_codes:
            yield Request(indexurl.format(index), self.dual_index, meta={'code': index})
            break


    def dualamount(self,x):
        if x[-1] in ['亿', '万']:
            amount = float(x[0:-1])
            amount_unit = x[-1]
            if amount_unit == '亿':
                amount *= amount * 1e8
            elif amount_unit == '万':
                amount *= amount * 1e4
        else:
            amount = float(x)
        return amount

    def dual(self,response):
        vody = json.loads(response.body[1:-1], encoding='utf-8')['info']
        zz = {'open': vody['o'], 'close': vody['c'], 'high': vody['h'], 'low': vody['l'], 'amount': vody['a'],
              'volume': vody['v'], 'trade': True if vody['a'] != '0' else False, 'code': response.meta['code'][:-1],
              'date': vody['time'][:10]}
        self.db.daily_k_all.insert(zz)
        self.db.daily_k_250.insert(zz)
        url = 'http://pdfm.eastmoney.com/EM_UBG_PDTI_Fast/api/js?rtntype=1&token=4f1862fc3b5e77c150a2b985b12db0fd&id={0}&type=k&authorityType={1}&cb='
        yield Request(url.format(response.meta['code'],'ba'),self.dualnext,meta={'code':response.meta['code']})

    def dualnext(self, response):

        body = str(response.body, encoding='utf-8')
        vody = body.split('\n')[-2][:-1].split(',')
        print (vody)
        zz = {'open': vody[1], 'close': vody[2], 'high': vody[3], 'low': vody[4], 'amount': self.dualamount(vody[6]),
              'volume': vody[5], 'trade': True if vody[5] != '0' else False, 'code': response.meta['code'][:-1],
              'date': vody[0][:10]}
        self.db.daily_kba_all.insert(zz)
        self.db.daily_kba_250.insert(zz)

    def dual_index(self,response):
        body = str(response.body, encoding='utf-8')[1:]
        body = body.split('\n')
        a = pd.DataFrame(body, )
        a = a[0].str.split(',', expand=True)
        a.columns = ['date', 'open', 'close', 'high', 'low', 'volume', 'amount', 'index', 'unknown']
        a['volume'] = a['volume'].apply(lambda x: int(x) * 100)
        a['amount'] = a['amount'].apply(self.dualamount).astype('int64')
        a['trade'] = a['volume'].apply(lambda x: True if x > 0 else False)
        a['code'] = response.meta['code']
        a = pd.DataFrame(a, columns=['date', 'open', 'close', 'high', 'low', 'amount', 'code'])
        a_250 = a.sort_values(by=['date'], ascending=False).head(1)
        self.db.daily_index_all.insert(json.loads((a.T.to_json())).values())
        self.db.daily_index_250.insert(json.loads((a_250.T.to_json())).values())